package main

import (
	"fmt"
	"github.com/gogf/gf/os/gtime"
	"log"
	newsModel "myblog/app/model/module/news"
	"net/http"
	"github.com/PuerkitoBio/goquery"
	"strconv"
	"sync"
)

type Res struct {
	Title   string
	Content string
}

var (
	// 存放图片链接的数据管道
	chanResult chan Res
	waitGroup  sync.WaitGroup
	// 用于监控协程
	chanTask chan string
)

func main() {
	//初始化管道
	chanResult = make(chan Res, 105)
	chanTask = make(chan string, 105)
	for i := 1; i < 106; i++ {
		waitGroup.Add(1)
		GetOnePage(i)
	}
	waitGroup.Add(1)
	go CheckOK()
	for v := range chanResult {
		Insertdata(v)
	}
	waitGroup.Wait()

}

//获取单个页面内容
func GetOnePage(num int) {
	// Request the HTML page.
	res, err := http.Get("http://www.topgoer.cn/docs/gomianshiti/mian" + strconv.Itoa(num))
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
	}

	// Load the HTML document
	doc, err := goquery.NewDocumentFromReader(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	// Find the review items
	doc.Find(".manual-right .manual-article ").Each(func(i int, s *goquery.Selection) {
		// For each item found, get the title
		title := s.Find("#article-title").Text()
		content, err := s.Find("#page-content .markdown-article-inner").Html()
		if err != nil {
			fmt.Println(err)
		}
		temp := Res{
			Title:   title,
			Content: content,
		}
		chanResult <- temp
		chanTask <- title
	})

}

// 任务统计协程
func CheckOK() {
	var count int
	for {
		title := <-chanTask
		fmt.Printf("%s 完成了爬取任务\n", title)
		count++
		if count == 105 {
			close(chanResult)
			break
		}
	}
	waitGroup.Done()
}

//插入数据
func Insertdata(res Res) {
	temp := newsModel.Entity{
		Catid:      33,
		Author:     "皓云",
		Title:      res.Title,
		Content:    res.Content,
		CreateTime: gtime.Now(),
	}

	newsModel.Insert(temp)
}
